package ca.uwindsor.cs.deepweb.utility.gov2;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.ArrayList;

import org.apache.lucene.index.IndexWriter;

import ca.uwindsor.cs.deepweb.utility.FileSizeDistributionAnalyzer;
import ca.uwindsor.cs.deepweb.utility.NekoHTMLHandler;

public class DocumentSizeReaderforGOV2Subset {

	private int globalID;

	protected IndexWriter indexWriter;

	protected FileSizeDistributionAnalyzer analyzer;

	protected int indexnum;

	protected int idendswith;

	/**
	 * Constructor
	 */
	public DocumentSizeReaderforGOV2Subset(int indexnum, int indends) {
		analyzer = new FileSizeDistributionAnalyzer(1000, null);
		globalID = 1;
		this.indexnum = indexnum;
		idendswith = indends;
	}

	protected long getFileNumericID(File file) {
		long id = Long.valueOf(file.getName().replaceAll("[^\\d]", ""));
		return id;
	}

	/**
	 * Start to create the index to store file size
	 * 
	 * @param indexdir
	 *            Where to store the index
	 * @param docdir
	 *            Where are the documents locate
	 */
	public void run(String indexdir, File docdir) {
		try {
			run(docdir);
			ArrayList<Integer> distribution = analyzer.getDistribution();
			StringBuffer result = new StringBuffer();
			result.append("less than or equals to 1k letter");
			result.append("\t");
			result.append(distribution.get(0).toString());
			result.append("\n");
			for (int i = 1; i < distribution.size(); i++) {
				distribution.set(i, new Integer(distribution.get(i)
						+ distribution.get(i - 1)));
				result.append("less than or equals to " + (i + 1) + "k letter");
				result.append("\t");
				result.append(distribution.get(i));
				result.append("\n");
			}
			System.out.println(result);
			System.out.println(globalID + " files are indexed!");
		} catch (IOException e) {
			e.printStackTrace();
			System.exit(1);
		}

	}

	protected void run(File dir) throws IOException {
		File[] dataFiles = dir.listFiles();

		// Add documents to the index
		for (int i = 0; i < dataFiles.length; i++) {
			File file = dataFiles[i];
			if (file.isFile()) {
				if (file.length() > 2048 && file.getName().endsWith("html")) {
					long id = getFileNumericID(file);
					int index = new Long(id % indexnum).intValue();
					if (idendswith != -1 && index != idendswith) {
						continue;
					} else {
						NekoHTMLHandler handler = new NekoHTMLHandler("latin1");
						try {
							handler.processContent(new FileInputStream(
									dataFiles[i]));
						} catch (Exception e) {
							// e.printStackTrace();
							System.err.println("Error on parsing: "
									+ dataFiles[i].getCanonicalPath());
							continue;
						}
						String content = handler.getContent().replaceAll(
								"\\s+", " ");
						if (content.length() < 2048) {
							continue;
						}

						analyzer.addSize(content.length());
						// txtReader.close();
						globalID++;
					}
				}
			} else if (file.isDirectory()) {
				run(dataFiles[i]);
			}
		}

	}

	/**
	 * @param args
	 */
	public static void main(String[] args) {
		
		if (args.length == 3) {
			int count = Integer.parseInt(args[1]);
			int sufix = Integer.parseInt(args[2]);
			DocumentSizeReaderforGOV2Subset index = new DocumentSizeReaderforGOV2Subset(count,sufix);
			index.run(null, new File(args[0]));
		} else {
			System.out
					.println("usage: java -jar DocumentSizeReaderforGOV2Subset.jar documentDir IndexCount idendswith");
		}
	}

}
